#!/bin/bash

top="$(dirname $(readlink -f $0))/.."
source $top/etc/build_settings

mkdir -p $data_dir $build_dir $index_dir

if [ ! -f "$corpus" ]; then 
	wget -O - $corpus_url | gzip -cd > $corpus
	[ "$?" != "0" ] && exit 1 
fi

if [ ! -f "$number_trailing" ];then
	wget -O "$number_trailing" "$number_trailing_url" || exit 1
fi

[ ! -f "$user_combine" ] && touch $user_combine

# Build Lexicon
cat $user_combine | sed -e 's/^/1 /g' > ${build_dir}/user_combine.txt || exit 1
cat $number_trailing | sed -e 's/^/1 /g' > ${build_dir}/number-trailing.txt || exit 1

$bin/pdc_normalize -v ${corpus} > ${build_dir}/normalized.txt || exit 1
$bin/ngm_tool -n1 -v ${build_dir}/normalized.txt > ${build_dir}/unigram.txt || exit 1
$bin/lexicon -v -b -i ${index_dir}/unigram.idx -s ${build_dir}/unigram.txt || exit 1
$bin/lexicon -v -b -i ${index_dir}/user_combine.idx -s ${build_dir}/user_combine.txt || exit 1
$bin/lexicon -v -b -i ${index_dir}/number-trailing.idx -s ${build_dir}/number-trailing.txt || exit 1


# Train CRF
echo 
echo "Training CRF Model may take a very long time (5 to 12 hours depend on your system)"
echo -n "Are you sure you wanna build (y/n)?"
read choice
if [ "$choice" == "y" ]; then
	crf_learn=$(which crf_learn)
	if [ -z "$crf_learn" ]; then
		echo "crf_learn not found!"
		exit
	fi
	$bin/crf_tool -v ${build_dir}/normalized.txt > ${build_dir}/crf.training.txt || exit 1
	# Train CRF1
	crf_learn $data_dir/crf.tmpl ${build_dir}/crf.training.txt ${index_dir}/crf1.model || exit 1
fi
